import random
import time
import requests as rs
from bs4 import BeautifulSoup as bs
import base64
import MySQLdb

def parse():
    url = 'https://www.dushu.com/lianzai/1115_1.html'
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36'
    }
    flag = True
    count = 0

    while flag:
        allText = []
        response = rs.get(url, headers=headers)
        html = response.content.decode()
        soup = bs(html, "lxml")
        lists = soup.select(".bookslist ul li")
        for i in lists:
            img_url = i.find("img").get('data-original')
            name = i.find_all('a')[1].string
            zz = i.find_all('p')[0].string
            text = i.find('p', {'class': 'disc eps'}).string
            oneText = [img_url, name, zz, text]
            allText.append(oneText)
            print(img_url, name, zz, text)

        try:
            if count == 0:
                next_url = 'https://www.dushu.com/lianzai/1115_2.html'
                url = next_url
                count += 1
            else:
                next_url = soup.find_all('a', {'class': 'next'})[0].get('href')
                if next_url and "javascript" not in next_url:
                    url = 'https://www.dushu.com' + next_url
                    print(url)
                else:
                    flag = False
        except:
            flag = False
        wirtieFile(allText)

def download_image(url):
    try:
        response = rs.get(url)
        if response.status_code == 200:
            return response.content
        else:
            return None
    except:
        pass

def wirtieFile(allText):
    print("Writing to file...")  # 添加此行
    with open('图书信息.txt', 'a', encoding='UTF-8') as f:
        for i in allText:
            print("Processing:", i)  # 添加此行
            img_data = download_image(i[0])
            if img_data:
                img_data_base64 = base64.b64encode(img_data).decode("utf-8")
            else:
                img_data_base64 = i[0]  # 将URL直接写入文件
            one = img_data_base64 + ',' + str(i[1]) + ',' + str(i[2]) + ',' + str(i[3]) + '\n'
            f.write(one)


# def saveToMysql(allText):
#     db = MySQLdb.connect('localhost', 'root', 'root', 'douban', charset='utf8')
#     # 使用cursor()方法获取游标
#     cursor = db.cursor()
#     '''插入'''
#     for one in allText:
#         # SQL语句
#         sql = "insert into booktest(book_img_url,book_name,book_zz,book_text) values('%s','%s','%s','%s')" % (one[0], one[1], one[2],one[3])
#         try:
#             # 执行SQL语句
#             cursor.execute(sql)
#             db.commit()
#         except:
#             db.rollback()
#
#     db.close()


if __name__ == '__main__':
    parse()

